{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use the keras-rl library to implement deep Q-learning \n",
    "\n",
    "https://github.com/keras-rl/keras-rl\n",
    "\n",
    "\n",
    "### Installation\n",
    "\n",
    "Install Keras-RL from Pypi (recommended):\n",
    "```\n",
    "pip install keras-rl\n",
    "```\n",
    "\n",
    "\n",
    "Install from Github source:\n",
    "```\n",
    "git clone https://github.com/keras-rl/keras-rl.git\n",
    "cd keras-rl\n",
    "python setup.py install\n",
    "````"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Flatten\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "from rl.agents.dqn import DQNAgent\n",
    "from rl.policy import EpsGreedyQPolicy\n",
    "from rl.memory import SequentialMemory\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENV_NAME = 'CartPole-v0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the environment and extract the number of actions.\n",
    "env = gym.make(ENV_NAME)\n",
    "np.random.seed(123)\n",
    "env.seed(123)\n",
    "nb_actions = env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape=(1,) + env.observation_space.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_17 (Flatten)         (None, 4)                 0         \n",
      "_________________________________________________________________\n",
      "dense_54 (Dense)             (None, 16)                80        \n",
      "_________________________________________________________________\n",
      "activation_54 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_55 (Dense)             (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "activation_55 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_56 (Dense)             (None, 16)                272       \n",
      "_________________________________________________________________\n",
      "activation_56 (Activation)   (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_57 (Dense)             (None, 2)                 34        \n",
      "_________________________________________________________________\n",
      "activation_57 (Activation)   (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 658\n",
      "Trainable params: 658\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Flatten(input_shape=(1,) + env.observation_space.shape))\n",
    "model.add(Dense(16))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(16))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(16))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(nb_actions))\n",
    "model.add(Activation('linear'))\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 1, 16)             80        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 1, 16)             0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1, 16)             272       \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 1, 16)             0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1, 16)             272       \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 1, 16)             0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 2)                 34        \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 658\n",
      "Trainable params: 658\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "# model.add(Flatten(input_shape=(1,4)))\n",
    "model.add(Dense(16, input_shape=(1,4)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(16))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(16))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(nb_actions))\n",
    "model.add(Activation('linear'))\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and\n",
    "# even the metrics!\n",
    "memory = SequentialMemory(limit=50000, window_length=1)\n",
    "policy = EpsGreedyQPolicy()\n",
    "dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=10,\n",
    "               target_model_update=1e-2, policy=policy)\n",
    "dqn.compile(Adam(lr=1e-3), metrics=['mae'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 10000 steps ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hongtao/anaconda3/envs/spinningup/lib/python3.6/site-packages/rl/memory.py:39: UserWarning: Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!\n",
      "  warnings.warn('Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   12/10000: episode: 1, duration: 3.960s, episode steps: 12, steps per second: 3, episode reward: 12.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.667 [0.000, 1.000], mean observation: -0.114 [-1.751, 0.973], loss: 0.497553, mean_absolute_error: 0.509256, mean_q: -0.016498\n",
      "   25/10000: episode: 2, duration: 0.149s, episode steps: 13, steps per second: 87, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.923 [0.000, 1.000], mean observation: -0.074 [-3.253, 2.194], loss: 0.440938, mean_absolute_error: 0.486571, mean_q: 0.067862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hongtao/anaconda3/envs/spinningup/lib/python3.6/site-packages/rl/memory.py:39: UserWarning: Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!\n",
      "  warnings.warn('Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!')\n",
      "/Users/hongtao/anaconda3/envs/spinningup/lib/python3.6/site-packages/rl/memory.py:39: UserWarning: Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!\n",
      "  warnings.warn('Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   34/10000: episode: 3, duration: 0.121s, episode steps: 9, steps per second: 74, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.127 [-2.809, 1.812], loss: 0.323152, mean_absolute_error: 0.426584, mean_q: 0.237247\n",
      "   44/10000: episode: 4, duration: 0.118s, episode steps: 10, steps per second: 85, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.157 [-3.116, 1.934], loss: 0.234744, mean_absolute_error: 0.355153, mean_q: 0.432159\n",
      "   52/10000: episode: 5, duration: 0.098s, episode steps: 8, steps per second: 82, episode reward: 8.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.138 [-2.563, 1.604], loss: 0.185758, mean_absolute_error: 0.309132, mean_q: 0.622192\n",
      "   61/10000: episode: 6, duration: 0.115s, episode steps: 9, steps per second: 78, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.124 [-2.762, 1.768], loss: 0.158073, mean_absolute_error: 0.291788, mean_q: 0.785875\n",
      "   70/10000: episode: 7, duration: 0.109s, episode steps: 9, steps per second: 82, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.139 [-2.798, 1.804], loss: 0.138914, mean_absolute_error: 0.292803, mean_q: 0.906254\n",
      "   79/10000: episode: 8, duration: 0.111s, episode steps: 9, steps per second: 81, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.138 [-2.817, 1.757], loss: 0.138887, mean_absolute_error: 0.328466, mean_q: 1.027576\n",
      "   88/10000: episode: 9, duration: 0.111s, episode steps: 9, steps per second: 81, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.118 [-2.741, 1.805], loss: 0.123793, mean_absolute_error: 0.360581, mean_q: 1.116737\n",
      "   99/10000: episode: 10, duration: 0.126s, episode steps: 11, steps per second: 87, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.909 [0.000, 1.000], mean observation: -0.136 [-2.749, 1.744], loss: 0.085117, mean_absolute_error: 0.386717, mean_q: 1.229742\n",
      "  108/10000: episode: 11, duration: 0.114s, episode steps: 9, steps per second: 79, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.143 [-2.802, 1.734], loss: 0.061897, mean_absolute_error: 0.438897, mean_q: 1.445295\n",
      "  117/10000: episode: 12, duration: 0.123s, episode steps: 9, steps per second: 73, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.136 [-2.795, 1.802], loss: 0.055966, mean_absolute_error: 0.505676, mean_q: 1.509498\n",
      "  126/10000: episode: 13, duration: 0.111s, episode steps: 9, steps per second: 81, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.144 [-2.881, 1.804], loss: 0.046306, mean_absolute_error: 0.582813, mean_q: 1.573691\n",
      "  135/10000: episode: 14, duration: 0.111s, episode steps: 9, steps per second: 81, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.889 [0.000, 1.000], mean observation: -0.126 [-2.230, 1.388], loss: 0.042492, mean_absolute_error: 0.629765, mean_q: 1.566971\n",
      "  145/10000: episode: 15, duration: 0.131s, episode steps: 10, steps per second: 76, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.900 [0.000, 1.000], mean observation: -0.142 [-2.486, 1.559], loss: 0.037709, mean_absolute_error: 0.690687, mean_q: 1.619794\n",
      "  155/10000: episode: 16, duration: 0.124s, episode steps: 10, steps per second: 81, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.132 [-3.016, 1.962], loss: 0.041534, mean_absolute_error: 0.751954, mean_q: 1.621642\n",
      "  164/10000: episode: 17, duration: 0.120s, episode steps: 9, steps per second: 75, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.167 [-2.873, 1.780], loss: 0.040052, mean_absolute_error: 0.815021, mean_q: 1.669149\n",
      "  173/10000: episode: 18, duration: 0.121s, episode steps: 9, steps per second: 75, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 1.000 [1.000, 1.000], mean observation: -0.152 [-2.835, 1.751], loss: 0.036849, mean_absolute_error: 0.873355, mean_q: 1.737483\n",
      "  182/10000: episode: 19, duration: 0.116s, episode steps: 9, steps per second: 78, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.889 [0.000, 1.000], mean observation: -0.116 [-2.436, 1.588], loss: 0.024223, mean_absolute_error: 0.842290, mean_q: 1.593807\n",
      "  193/10000: episode: 20, duration: 0.170s, episode steps: 11, steps per second: 65, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.727 [0.000, 1.000], mean observation: -0.118 [-1.921, 1.182], loss: 0.014280, mean_absolute_error: 0.834201, mean_q: 1.609290\n",
      "  204/10000: episode: 21, duration: 0.160s, episode steps: 11, steps per second: 69, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.727 [0.000, 1.000], mean observation: -0.091 [-2.058, 1.388], loss: 0.010265, mean_absolute_error: 0.854559, mean_q: 1.694313\n",
      "  215/10000: episode: 22, duration: 0.153s, episode steps: 11, steps per second: 72, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.727 [0.000, 1.000], mean observation: -0.119 [-1.760, 1.160], loss: 0.011508, mean_absolute_error: 0.873250, mean_q: 1.689020\n",
      "  227/10000: episode: 23, duration: 0.188s, episode steps: 12, steps per second: 64, episode reward: 12.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.667 [0.000, 1.000], mean observation: -0.091 [-1.735, 1.207], loss: 0.013702, mean_absolute_error: 0.916551, mean_q: 1.764569\n",
      "  238/10000: episode: 24, duration: 0.140s, episode steps: 11, steps per second: 79, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.727 [0.000, 1.000], mean observation: -0.118 [-2.102, 1.382], loss: 0.014939, mean_absolute_error: 0.938211, mean_q: 1.830701\n",
      "  248/10000: episode: 25, duration: 0.168s, episode steps: 10, steps per second: 60, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.700 [0.000, 1.000], mean observation: -0.137 [-1.531, 0.963], loss: 0.013365, mean_absolute_error: 0.955171, mean_q: 1.846092\n",
      "  261/10000: episode: 26, duration: 0.204s, episode steps: 13, steps per second: 64, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.615 [0.000, 1.000], mean observation: -0.101 [-1.704, 1.186], loss: 0.019122, mean_absolute_error: 1.012261, mean_q: 1.954772\n",
      "  270/10000: episode: 27, duration: 0.158s, episode steps: 9, steps per second: 57, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.778 [0.000, 1.000], mean observation: -0.148 [-1.876, 1.164], loss: 0.022267, mean_absolute_error: 1.028419, mean_q: 1.979342\n",
      "  285/10000: episode: 28, duration: 0.252s, episode steps: 15, steps per second: 59, episode reward: 15.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.600 [0.000, 1.000], mean observation: -0.104 [-1.420, 0.927], loss: 0.019197, mean_absolute_error: 1.095426, mean_q: 2.108922\n",
      "  295/10000: episode: 29, duration: 0.175s, episode steps: 10, steps per second: 57, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.100 [0.000, 1.000], mean observation: 0.161 [-1.521, 2.531], loss: 0.021881, mean_absolute_error: 1.156726, mean_q: 2.196498\n",
      "  306/10000: episode: 30, duration: 0.183s, episode steps: 11, steps per second: 60, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.091 [0.000, 1.000], mean observation: 0.125 [-1.717, 2.811], loss: 0.047369, mean_absolute_error: 1.208306, mean_q: 2.292123\n",
      "  316/10000: episode: 31, duration: 0.176s, episode steps: 10, steps per second: 57, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.200 [0.000, 1.000], mean observation: 0.113 [-1.593, 2.366], loss: 0.120569, mean_absolute_error: 1.299114, mean_q: 2.372253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  342/10000: episode: 32, duration: 0.430s, episode steps: 26, steps per second: 61, episode reward: 26.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.231 [0.000, 1.000], mean observation: -0.021 [-2.750, 3.620], loss: 0.096825, mean_absolute_error: 1.366028, mean_q: 2.531889\n",
      "  352/10000: episode: 33, duration: 0.189s, episode steps: 10, steps per second: 53, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.100 [0.000, 1.000], mean observation: 0.148 [-1.544, 2.534], loss: 0.098142, mean_absolute_error: 1.481573, mean_q: 2.840460\n",
      "  361/10000: episode: 34, duration: 0.170s, episode steps: 9, steps per second: 53, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.111 [0.000, 1.000], mean observation: 0.145 [-1.322, 2.257], loss: 0.196824, mean_absolute_error: 1.524436, mean_q: 2.806054\n",
      "  369/10000: episode: 35, duration: 0.151s, episode steps: 8, steps per second: 53, episode reward: 8.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.144 [-1.590, 2.565], loss: 0.168530, mean_absolute_error: 1.564705, mean_q: 2.856247\n",
      "  379/10000: episode: 36, duration: 0.195s, episode steps: 10, steps per second: 51, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.134 [-1.977, 3.069], loss: 0.297060, mean_absolute_error: 1.655295, mean_q: 3.016542\n",
      "  389/10000: episode: 37, duration: 0.159s, episode steps: 10, steps per second: 63, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.135 [-1.937, 3.009], loss: 0.200282, mean_absolute_error: 1.637776, mean_q: 2.965510\n",
      "  397/10000: episode: 38, duration: 0.135s, episode steps: 8, steps per second: 59, episode reward: 8.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.143 [-1.547, 2.531], loss: 0.206478, mean_absolute_error: 1.655385, mean_q: 2.992626\n",
      "  407/10000: episode: 39, duration: 0.182s, episode steps: 10, steps per second: 55, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.137 [-1.958, 3.050], loss: 0.208824, mean_absolute_error: 1.735142, mean_q: 3.197716\n",
      "  416/10000: episode: 40, duration: 0.143s, episode steps: 9, steps per second: 63, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.144 [-1.774, 2.819], loss: 0.195193, mean_absolute_error: 1.773987, mean_q: 3.302514\n",
      "  424/10000: episode: 41, duration: 0.122s, episode steps: 8, steps per second: 65, episode reward: 8.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.169 [-1.565, 2.578], loss: 0.240618, mean_absolute_error: 1.818421, mean_q: 3.361008\n",
      "  435/10000: episode: 42, duration: 0.188s, episode steps: 11, steps per second: 59, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.091 [0.000, 1.000], mean observation: 0.106 [-1.809, 2.808], loss: 0.218322, mean_absolute_error: 1.850535, mean_q: 3.424123\n",
      "  445/10000: episode: 43, duration: 0.186s, episode steps: 10, steps per second: 54, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.200 [0.000, 1.000], mean observation: 0.162 [-1.336, 2.312], loss: 0.183743, mean_absolute_error: 1.823935, mean_q: 3.364591\n",
      "  455/10000: episode: 44, duration: 0.192s, episode steps: 10, steps per second: 52, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.159 [-1.915, 3.070], loss: 0.207694, mean_absolute_error: 1.961151, mean_q: 3.736166\n",
      "  465/10000: episode: 45, duration: 0.161s, episode steps: 10, steps per second: 62, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.100 [0.000, 1.000], mean observation: 0.154 [-1.528, 2.588], loss: 0.285569, mean_absolute_error: 1.965652, mean_q: 3.581646\n",
      "  474/10000: episode: 46, duration: 0.150s, episode steps: 9, steps per second: 60, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.187 [-1.727, 2.899], loss: 0.232855, mean_absolute_error: 2.009974, mean_q: 3.831956\n",
      "  483/10000: episode: 47, duration: 0.185s, episode steps: 9, steps per second: 49, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.164 [-1.721, 2.789], loss: 0.270802, mean_absolute_error: 2.042488, mean_q: 3.803957\n",
      "  493/10000: episode: 48, duration: 0.203s, episode steps: 10, steps per second: 49, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.200 [0.000, 1.000], mean observation: 0.113 [-1.212, 1.985], loss: 0.253024, mean_absolute_error: 2.058556, mean_q: 3.844785\n",
      "  505/10000: episode: 49, duration: 0.252s, episode steps: 12, steps per second: 48, episode reward: 12.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.167 [0.000, 1.000], mean observation: 0.110 [-1.575, 2.545], loss: 0.250075, mean_absolute_error: 2.126002, mean_q: 4.100665\n",
      "  515/10000: episode: 50, duration: 0.170s, episode steps: 10, steps per second: 59, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.200 [0.000, 1.000], mean observation: 0.110 [-1.584, 2.364], loss: 0.259643, mean_absolute_error: 2.139257, mean_q: 4.045831\n",
      "  525/10000: episode: 51, duration: 0.185s, episode steps: 10, steps per second: 54, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.151 [-1.922, 3.109], loss: 0.228265, mean_absolute_error: 2.167935, mean_q: 4.080089\n",
      "  536/10000: episode: 52, duration: 0.194s, episode steps: 11, steps per second: 57, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.182 [0.000, 1.000], mean observation: 0.123 [-1.597, 2.422], loss: 0.265342, mean_absolute_error: 2.227136, mean_q: 4.060940\n",
      "  546/10000: episode: 53, duration: 0.197s, episode steps: 10, steps per second: 51, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.132 [-1.994, 3.014], loss: 0.227026, mean_absolute_error: 2.242022, mean_q: 4.244925\n",
      "  555/10000: episode: 54, duration: 0.179s, episode steps: 9, steps per second: 50, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.135 [-1.765, 2.769], loss: 0.253911, mean_absolute_error: 2.273658, mean_q: 4.330843\n",
      "  564/10000: episode: 55, duration: 0.163s, episode steps: 9, steps per second: 55, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.145 [-1.742, 2.813], loss: 0.185570, mean_absolute_error: 2.273664, mean_q: 4.346209\n",
      "  575/10000: episode: 56, duration: 0.184s, episode steps: 11, steps per second: 60, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.091 [0.000, 1.000], mean observation: 0.123 [-1.790, 2.815], loss: 0.268499, mean_absolute_error: 2.310026, mean_q: 4.377471\n",
      "  584/10000: episode: 57, duration: 0.163s, episode steps: 9, steps per second: 55, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.143 [-1.775, 2.809], loss: 0.251009, mean_absolute_error: 2.362797, mean_q: 4.584296\n",
      "  594/10000: episode: 58, duration: 0.183s, episode steps: 10, steps per second: 55, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.154 [-1.927, 3.118], loss: 0.247240, mean_absolute_error: 2.388568, mean_q: 4.587520\n",
      "  602/10000: episode: 59, duration: 0.149s, episode steps: 8, steps per second: 54, episode reward: 8.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.140 [-1.605, 2.571], loss: 0.258900, mean_absolute_error: 2.404877, mean_q: 4.570201\n",
      "  612/10000: episode: 60, duration: 0.200s, episode steps: 10, steps per second: 50, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.000 [0.000, 0.000], mean observation: 0.149 [-1.997, 3.097], loss: 0.215756, mean_absolute_error: 2.413985, mean_q: 4.621381\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  623/10000: episode: 61, duration: 0.230s, episode steps: 11, steps per second: 48, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.182 [0.000, 1.000], mean observation: 0.107 [-1.385, 2.240], loss: 0.252719, mean_absolute_error: 2.443873, mean_q: 4.652440\n",
      "  636/10000: episode: 62, duration: 0.228s, episode steps: 13, steps per second: 57, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.154 [0.000, 1.000], mean observation: 0.097 [-1.756, 2.800], loss: 0.248125, mean_absolute_error: 2.526170, mean_q: 4.827978\n",
      "  647/10000: episode: 63, duration: 0.242s, episode steps: 11, steps per second: 45, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.091 [0.000, 1.000], mean observation: 0.135 [-1.718, 2.705], loss: 0.179364, mean_absolute_error: 2.546573, mean_q: 4.927422\n",
      "  657/10000: episode: 64, duration: 0.221s, episode steps: 10, steps per second: 45, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.100 [0.000, 1.000], mean observation: 0.140 [-1.717, 2.720], loss: 0.223092, mean_absolute_error: 2.519887, mean_q: 4.811268\n",
      "  668/10000: episode: 65, duration: 0.229s, episode steps: 11, steps per second: 48, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.182 [0.000, 1.000], mean observation: 0.102 [-1.578, 2.414], loss: 0.230585, mean_absolute_error: 2.599591, mean_q: 4.989634\n",
      "  678/10000: episode: 66, duration: 0.212s, episode steps: 10, steps per second: 47, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.200 [0.000, 1.000], mean observation: 0.123 [-1.352, 2.134], loss: 0.229576, mean_absolute_error: 2.603299, mean_q: 4.916408\n",
      "  691/10000: episode: 67, duration: 0.322s, episode steps: 13, steps per second: 40, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.308 [0.000, 1.000], mean observation: 0.093 [-1.370, 2.041], loss: 0.176493, mean_absolute_error: 2.709732, mean_q: 5.178645\n",
      "  701/10000: episode: 68, duration: 0.191s, episode steps: 10, steps per second: 52, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.300 [0.000, 1.000], mean observation: 0.126 [-0.961, 1.665], loss: 0.216088, mean_absolute_error: 2.653049, mean_q: 4.929430\n",
      "  736/10000: episode: 69, duration: 0.788s, episode steps: 35, steps per second: 44, episode reward: 35.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.486 [0.000, 1.000], mean observation: -0.088 [-0.952, 0.601], loss: 0.176963, mean_absolute_error: 2.758446, mean_q: 5.209208\n",
      "  768/10000: episode: 70, duration: 0.680s, episode steps: 32, steps per second: 47, episode reward: 32.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.469 [0.000, 1.000], mean observation: -0.084 [-0.714, 0.222], loss: 0.206260, mean_absolute_error: 2.819791, mean_q: 5.316781\n",
      "  790/10000: episode: 71, duration: 0.549s, episode steps: 22, steps per second: 40, episode reward: 22.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.500 [0.000, 1.000], mean observation: 0.080 [-0.453, 1.020], loss: 0.143682, mean_absolute_error: 2.975178, mean_q: 5.730409\n",
      "  815/10000: episode: 72, duration: 0.687s, episode steps: 25, steps per second: 36, episode reward: 25.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.480 [0.000, 1.000], mean observation: 0.111 [-0.197, 1.073], loss: 0.235535, mean_absolute_error: 3.062675, mean_q: 5.802898\n",
      "  844/10000: episode: 73, duration: 0.705s, episode steps: 29, steps per second: 41, episode reward: 29.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.483 [0.000, 1.000], mean observation: 0.036 [-0.837, 1.476], loss: 0.222289, mean_absolute_error: 3.157141, mean_q: 6.011464\n",
      "  870/10000: episode: 74, duration: 0.748s, episode steps: 26, steps per second: 35, episode reward: 26.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.500 [0.000, 1.000], mean observation: 0.120 [-0.206, 1.041], loss: 0.372700, mean_absolute_error: 3.228757, mean_q: 6.105584\n",
      "  893/10000: episode: 75, duration: 0.606s, episode steps: 23, steps per second: 38, episode reward: 23.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.478 [0.000, 1.000], mean observation: 0.087 [-0.383, 0.902], loss: 0.325753, mean_absolute_error: 3.317826, mean_q: 6.293920\n",
      "  941/10000: episode: 76, duration: 1.180s, episode steps: 48, steps per second: 41, episode reward: 48.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.458 [0.000, 1.000], mean observation: -0.071 [-0.726, 0.262], loss: 0.343608, mean_absolute_error: 3.444919, mean_q: 6.566545\n",
      " 1013/10000: episode: 77, duration: 2.306s, episode steps: 72, steps per second: 31, episode reward: 72.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.472 [0.000, 1.000], mean observation: -0.096 [-0.961, 0.796], loss: 0.298714, mean_absolute_error: 3.587648, mean_q: 6.855951\n",
      " 1034/10000: episode: 78, duration: 0.605s, episode steps: 21, steps per second: 35, episode reward: 21.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.524 [0.000, 1.000], mean observation: 0.105 [-0.360, 0.855], loss: 0.315850, mean_absolute_error: 3.806700, mean_q: 7.316886\n",
      " 1108/10000: episode: 79, duration: 2.408s, episode steps: 74, steps per second: 31, episode reward: 74.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.486 [0.000, 1.000], mean observation: -0.104 [-1.094, 0.458], loss: 0.396730, mean_absolute_error: 3.994033, mean_q: 7.679094\n",
      " 1164/10000: episode: 80, duration: 2.287s, episode steps: 56, steps per second: 24, episode reward: 56.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.464 [0.000, 1.000], mean observation: -0.145 [-0.863, 0.243], loss: 0.404512, mean_absolute_error: 4.202549, mean_q: 8.147038\n",
      " 1203/10000: episode: 81, duration: 1.336s, episode steps: 39, steps per second: 29, episode reward: 39.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.538 [0.000, 1.000], mean observation: 0.101 [-0.195, 0.599], loss: 0.509438, mean_absolute_error: 4.334435, mean_q: 8.371168\n",
      " 1332/10000: episode: 82, duration: 3.422s, episode steps: 129, steps per second: 38, episode reward: 129.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.496 [0.000, 1.000], mean observation: 0.203 [-1.067, 1.727], loss: 0.486339, mean_absolute_error: 4.710400, mean_q: 9.142510\n",
      " 1516/10000: episode: 83, duration: 5.089s, episode steps: 184, steps per second: 36, episode reward: 184.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.516 [0.000, 1.000], mean observation: 0.390 [-0.976, 2.307], loss: 0.429089, mean_absolute_error: 5.195320, mean_q: 10.168212\n",
      " 1655/10000: episode: 84, duration: 3.010s, episode steps: 139, steps per second: 46, episode reward: 139.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.540 [0.000, 1.000], mean observation: 0.353 [-0.465, 2.033], loss: 0.552532, mean_absolute_error: 5.823549, mean_q: 11.487599\n",
      " 1774/10000: episode: 85, duration: 2.535s, episode steps: 119, steps per second: 47, episode reward: 119.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.471 [0.000, 1.000], mean observation: -0.135 [-1.287, 0.415], loss: 0.546631, mean_absolute_error: 6.292055, mean_q: 12.468135\n",
      " 1872/10000: episode: 86, duration: 2.255s, episode steps: 98, steps per second: 43, episode reward: 98.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.459 [0.000, 1.000], mean observation: -0.189 [-1.446, 0.365], loss: 0.621248, mean_absolute_error: 6.676717, mean_q: 13.217762\n",
      " 1986/10000: episode: 87, duration: 2.585s, episode steps: 114, steps per second: 44, episode reward: 114.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: -0.144 [-1.227, 0.292], loss: 0.556783, mean_absolute_error: 7.079693, mean_q: 14.072614\n",
      " 2089/10000: episode: 88, duration: 2.189s, episode steps: 103, steps per second: 47, episode reward: 103.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.466 [0.000, 1.000], mean observation: -0.212 [-1.308, 0.650], loss: 0.751858, mean_absolute_error: 7.412617, mean_q: 14.771662\n",
      " 2264/10000: episode: 89, duration: 3.743s, episode steps: 175, steps per second: 47, episode reward: 175.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: -0.177 [-1.657, 0.310], loss: 0.616657, mean_absolute_error: 7.955935, mean_q: 15.899356\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 2406/10000: episode: 90, duration: 3.271s, episode steps: 142, steps per second: 43, episode reward: 142.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.472 [0.000, 1.000], mean observation: -0.204 [-1.414, 0.402], loss: 0.689873, mean_absolute_error: 8.452146, mean_q: 16.981213\n",
      " 2546/10000: episode: 91, duration: 3.013s, episode steps: 140, steps per second: 46, episode reward: 140.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.471 [0.000, 1.000], mean observation: -0.237 [-1.413, 0.595], loss: 0.652020, mean_absolute_error: 8.917190, mean_q: 18.000187\n",
      " 2696/10000: episode: 92, duration: 3.173s, episode steps: 150, steps per second: 47, episode reward: 150.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.473 [0.000, 1.000], mean observation: -0.253 [-1.483, 0.556], loss: 0.632249, mean_absolute_error: 9.474723, mean_q: 19.130997\n",
      " 2831/10000: episode: 93, duration: 2.989s, episode steps: 135, steps per second: 45, episode reward: 135.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.467 [0.000, 1.000], mean observation: -0.269 [-1.643, 0.513], loss: 0.644798, mean_absolute_error: 10.027730, mean_q: 20.292942\n",
      " 2983/10000: episode: 94, duration: 2.769s, episode steps: 152, steps per second: 55, episode reward: 152.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: -0.252 [-1.475, 0.560], loss: 0.586314, mean_absolute_error: 10.482678, mean_q: 21.169678\n",
      " 3183/10000: episode: 95, duration: 3.257s, episode steps: 200, steps per second: 61, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.520 [0.000, 1.000], mean observation: 0.281 [-0.711, 1.886], loss: 0.771326, mean_absolute_error: 11.092213, mean_q: 22.419439\n",
      " 3348/10000: episode: 96, duration: 2.654s, episode steps: 165, steps per second: 62, episode reward: 165.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.267 [-1.588, 0.793], loss: 0.885074, mean_absolute_error: 11.805822, mean_q: 23.939089\n",
      " 3482/10000: episode: 97, duration: 2.152s, episode steps: 134, steps per second: 62, episode reward: 134.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.470 [0.000, 1.000], mean observation: -0.315 [-1.608, 0.323], loss: 0.825048, mean_absolute_error: 12.332193, mean_q: 24.986940\n",
      " 3682/10000: episode: 98, duration: 3.184s, episode steps: 200, steps per second: 63, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.485 [0.000, 1.000], mean observation: -0.141 [-1.145, 0.704], loss: 0.884617, mean_absolute_error: 12.926129, mean_q: 26.212158\n",
      " 3872/10000: episode: 99, duration: 2.911s, episode steps: 190, steps per second: 65, episode reward: 190.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.228 [-1.638, 0.845], loss: 1.063510, mean_absolute_error: 13.601347, mean_q: 27.522343\n",
      " 4012/10000: episode: 100, duration: 2.654s, episode steps: 140, steps per second: 53, episode reward: 140.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.471 [0.000, 1.000], mean observation: -0.318 [-1.613, 0.445], loss: 0.937643, mean_absolute_error: 14.255636, mean_q: 28.952452\n",
      " 4155/10000: episode: 101, duration: 3.550s, episode steps: 143, steps per second: 40, episode reward: 143.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.476 [0.000, 1.000], mean observation: -0.313 [-1.668, 0.492], loss: 1.082548, mean_absolute_error: 14.630658, mean_q: 29.698273\n",
      " 4307/10000: episode: 102, duration: 4.416s, episode steps: 152, steps per second: 34, episode reward: 152.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: -0.304 [-1.710, 0.357], loss: 1.053803, mean_absolute_error: 15.282689, mean_q: 31.021606\n",
      " 4473/10000: episode: 103, duration: 4.647s, episode steps: 166, steps per second: 36, episode reward: 166.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.476 [0.000, 1.000], mean observation: -0.279 [-1.681, 0.516], loss: 0.922894, mean_absolute_error: 15.734209, mean_q: 31.977074\n",
      " 4673/10000: episode: 104, duration: 4.524s, episode steps: 200, steps per second: 44, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.490 [0.000, 1.000], mean observation: -0.077 [-0.926, 0.529], loss: 0.997594, mean_absolute_error: 16.163971, mean_q: 32.883270\n",
      " 4840/10000: episode: 105, duration: 4.006s, episode steps: 167, steps per second: 42, episode reward: 167.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.235 [-1.413, 0.494], loss: 1.360427, mean_absolute_error: 16.904472, mean_q: 34.294754\n",
      " 4996/10000: episode: 106, duration: 4.452s, episode steps: 156, steps per second: 35, episode reward: 156.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: -0.293 [-1.663, 0.413], loss: 0.923479, mean_absolute_error: 17.365913, mean_q: 35.308361\n",
      " 5196/10000: episode: 107, duration: 5.858s, episode steps: 200, steps per second: 34, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.480 [0.000, 1.000], mean observation: -0.220 [-1.586, 0.416], loss: 1.153954, mean_absolute_error: 17.930222, mean_q: 36.455353\n",
      " 5388/10000: episode: 108, duration: 5.142s, episode steps: 192, steps per second: 37, episode reward: 192.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.225 [-1.595, 0.444], loss: 1.576519, mean_absolute_error: 18.554817, mean_q: 37.587692\n",
      " 5588/10000: episode: 109, duration: 4.946s, episode steps: 200, steps per second: 40, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.480 [0.000, 1.000], mean observation: -0.212 [-1.603, 0.439], loss: 0.941452, mean_absolute_error: 19.064342, mean_q: 38.793724\n",
      " 5735/10000: episode: 110, duration: 2.997s, episode steps: 147, steps per second: 49, episode reward: 147.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.476 [0.000, 1.000], mean observation: -0.281 [-1.579, 0.448], loss: 1.222750, mean_absolute_error: 19.478548, mean_q: 39.682278\n",
      " 5888/10000: episode: 111, duration: 3.299s, episode steps: 153, steps per second: 46, episode reward: 153.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.477 [0.000, 1.000], mean observation: -0.268 [-1.644, 0.309], loss: 1.296724, mean_absolute_error: 20.069969, mean_q: 40.778992\n",
      " 6052/10000: episode: 112, duration: 3.552s, episode steps: 164, steps per second: 46, episode reward: 164.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.476 [0.000, 1.000], mean observation: -0.265 [-1.595, 0.513], loss: 1.405581, mean_absolute_error: 20.308460, mean_q: 41.219082\n",
      " 6252/10000: episode: 113, duration: 3.864s, episode steps: 200, steps per second: 52, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.490 [0.000, 1.000], mean observation: -0.103 [-0.894, 0.487], loss: 1.504496, mean_absolute_error: 20.728043, mean_q: 42.087296\n",
      " 6436/10000: episode: 114, duration: 3.256s, episode steps: 184, steps per second: 57, episode reward: 184.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.484 [0.000, 1.000], mean observation: -0.236 [-1.577, 0.730], loss: 1.438753, mean_absolute_error: 21.158022, mean_q: 43.072777\n",
      " 6633/10000: episode: 115, duration: 3.351s, episode steps: 197, steps per second: 59, episode reward: 197.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.482 [0.000, 1.000], mean observation: -0.220 [-1.628, 0.463], loss: 1.196517, mean_absolute_error: 21.829971, mean_q: 44.355885\n",
      " 6833/10000: episode: 116, duration: 3.243s, episode steps: 200, steps per second: 62, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.510 [0.000, 1.000], mean observation: 0.154 [-0.446, 1.083], loss: 1.411430, mean_absolute_error: 22.299932, mean_q: 45.246506\n",
      " 6978/10000: episode: 117, duration: 2.570s, episode steps: 145, steps per second: 56, episode reward: 145.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.476 [0.000, 1.000], mean observation: -0.278 [-1.515, 0.469], loss: 1.566139, mean_absolute_error: 22.653406, mean_q: 45.947895\n",
      " 7124/10000: episode: 118, duration: 2.486s, episode steps: 146, steps per second: 59, episode reward: 146.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.271 [-1.494, 0.907], loss: 1.572203, mean_absolute_error: 22.847200, mean_q: 46.418358\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 7265/10000: episode: 119, duration: 2.619s, episode steps: 141, steps per second: 54, episode reward: 141.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.482 [0.000, 1.000], mean observation: -0.306 [-1.822, 0.412], loss: 1.385589, mean_absolute_error: 23.075144, mean_q: 46.811882\n",
      " 7409/10000: episode: 120, duration: 2.593s, episode steps: 144, steps per second: 56, episode reward: 144.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.275 [-1.600, 0.569], loss: 1.875434, mean_absolute_error: 23.552090, mean_q: 47.844639\n",
      " 7549/10000: episode: 121, duration: 2.462s, episode steps: 140, steps per second: 57, episode reward: 140.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.486 [0.000, 1.000], mean observation: -0.297 [-1.682, 0.471], loss: 1.869521, mean_absolute_error: 24.073183, mean_q: 48.719387\n",
      " 7749/10000: episode: 122, duration: 4.474s, episode steps: 200, steps per second: 45, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.515 [0.000, 1.000], mean observation: 0.202 [-0.640, 1.385], loss: 1.794757, mean_absolute_error: 23.969093, mean_q: 48.475727\n",
      " 7883/10000: episode: 123, duration: 3.408s, episode steps: 134, steps per second: 39, episode reward: 134.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.470 [0.000, 1.000], mean observation: -0.313 [-1.673, 0.472], loss: 1.539937, mean_absolute_error: 24.531727, mean_q: 49.541504\n",
      " 8083/10000: episode: 124, duration: 5.841s, episode steps: 200, steps per second: 34, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.480 [0.000, 1.000], mean observation: -0.165 [-1.457, 0.583], loss: 1.121291, mean_absolute_error: 24.657339, mean_q: 50.045200\n",
      " 8279/10000: episode: 125, duration: 5.414s, episode steps: 196, steps per second: 36, episode reward: 196.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.485 [0.000, 1.000], mean observation: -0.211 [-1.511, 0.469], loss: 1.307432, mean_absolute_error: 25.174658, mean_q: 51.059826\n",
      " 8414/10000: episode: 126, duration: 3.973s, episode steps: 135, steps per second: 34, episode reward: 135.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.481 [0.000, 1.000], mean observation: -0.295 [-1.610, 0.538], loss: 1.935436, mean_absolute_error: 25.807823, mean_q: 52.160748\n",
      " 8576/10000: episode: 127, duration: 4.191s, episode steps: 162, steps per second: 39, episode reward: 162.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.475 [0.000, 1.000], mean observation: -0.251 [-1.606, 0.637], loss: 1.061363, mean_absolute_error: 25.737030, mean_q: 52.095764\n",
      " 8776/10000: episode: 128, duration: 4.979s, episode steps: 200, steps per second: 40, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.500 [0.000, 1.000], mean observation: 0.018 [-0.633, 0.537], loss: 1.991469, mean_absolute_error: 26.008266, mean_q: 52.526329\n",
      " 8976/10000: episode: 129, duration: 4.314s, episode steps: 200, steps per second: 46, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.480 [0.000, 1.000], mean observation: -0.157 [-1.480, 0.556], loss: 1.983715, mean_absolute_error: 26.327988, mean_q: 53.167389\n",
      " 9176/10000: episode: 130, duration: 3.792s, episode steps: 200, steps per second: 53, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.485 [0.000, 1.000], mean observation: -0.120 [-1.307, 0.716], loss: 2.027106, mean_absolute_error: 26.988396, mean_q: 54.443127\n",
      " 9333/10000: episode: 131, duration: 2.549s, episode steps: 157, steps per second: 62, episode reward: 157.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.478 [0.000, 1.000], mean observation: -0.273 [-1.853, 0.448], loss: 1.586850, mean_absolute_error: 27.123495, mean_q: 54.837280\n",
      " 9533/10000: episode: 132, duration: 3.031s, episode steps: 200, steps per second: 66, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.515 [0.000, 1.000], mean observation: 0.247 [-0.606, 1.756], loss: 1.602828, mean_absolute_error: 27.348833, mean_q: 55.255898\n",
      " 9733/10000: episode: 133, duration: 3.357s, episode steps: 200, steps per second: 60, episode reward: 200.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.495 [0.000, 1.000], mean observation: -0.039 [-0.905, 0.577], loss: 1.450481, mean_absolute_error: 27.698330, mean_q: 55.914707\n",
      " 9902/10000: episode: 134, duration: 2.783s, episode steps: 169, steps per second: 61, episode reward: 169.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.473 [0.000, 1.000], mean observation: -0.244 [-1.796, 0.480], loss: 1.645377, mean_absolute_error: 28.061737, mean_q: 56.606770\n",
      "done, took 219.311 seconds\n"
     ]
    }
   ],
   "source": [
    "H = dqn.fit(env, nb_steps=10000, visualize=False, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing for 5 episodes ...\n",
      "Episode 1: reward: 189.000, steps: 189\n",
      "Episode 2: reward: 169.000, steps: 169\n",
      "Episode 3: reward: 170.000, steps: 170\n",
      "Episode 4: reward: 184.000, steps: 184\n",
      "Episode 5: reward: 142.000, steps: 142\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb3289bbe0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dqn.test(env, nb_episodes=5, visualize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['episode_reward', 'nb_episode_steps', 'nb_steps'])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "H.history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0xb36f09ba8>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztvXl4Y2d59/95tNmS93U8i2c8k1mykMwkGbIHsgCBlJBA2QItoUBTytrlLQX661v6vu1bSoG2lBIIJAVaGgJZSJqmgRCyDSQhM8lkkjCLPYtnPIv3RbJ26fn9cc6RZVu7jmzZc3+uy5elR0fSc3Sk77nP/dyL0lojCIIgLF8ciz0BQRAEobKI0AuCICxzROgFQRCWOSL0giAIyxwRekEQhGWOCL0gCMIyR4ReEARhmSNCLwiCsMwRoRcEQVjmuBZ7AgDt7e26p6dnsachCIKwpNi1a9eI1roj33ZVIfQ9PT3s3LlzsachCIKwpFBK9ReynbhuBEEQljki9IIgCMscEXpBEIRljgi9IAjCMkeEXhAEYZmTV+iVUt1KqceVUnuVUq8qpT5tjrcqpR5VSvWa/1vMcaWU+ppSqk8ptUcpdUGld0IQBEHITiEWfRz4U631WcAlwMeVUmcDnwUe01pvAh4z7wO8Bdhk/t0K3Gb7rAVBEISCyRtHr7U+CZw0b/uVUnuB1cCNwFXmZt8DngD+3Bz/vjZ6FD6rlGpWSq00X0dYhvQNBTg1GeaKTe1FPzcaT3LPrgHeccFqat3OCswuOy8cHcepFFu7m0t6/qsnJhkJRHn95rz5KgDsP+Xn+ESQa85cUdL7zSWZ1Pxo5zHetm0VPk/pKTH9o9P86uAo79nejcOhbJlbPn7y4nGuPrOTJq8753bPHxkjmdRcvKHNtvceGA/y450DFNNGtcbt5IOX9VBXk/tzjieS/Nsvj+APx+Y95nAobr5oLSsaa4uec7kU9e1QSvUA5wPPASss8dZan1RKdZqbrQaOpT1twBybJfRKqVsxLH7Wrl1bwtSFauGfH+vll30jvPCXbyz6ub/sG+Hz97/MkD/MH71hcwVml53P3/cyXo+T+z92eUnP/4ef7mdH7wj3/OFlbMtzshj2R/idO55Da83O/6/4zykTLx6b4LP3vcxEKMZHX39G0c8/OhrkX37Ry30vHieR1Gxe0cCF61psmVsujo0F+aO7d/M3N72G37lkXc5t//zePRwanuZ9F6/lL64/K6/QFsL3fnWEbz99GFXgOc06H6xt9XHD1lU5t33h6AR/+/BegHmvrzUkNfzJGxf2ew5FCL1Sqh64F/gjrfWUyv4pZXpg3qlTa307cDvA9u3bpUP5EuboWJCx6SiTwRhNvtwW2lyG/GEAvvXkId7z2m5WNnkrMcV5RONJ+oYC+DxOtNbk+D5npX80SDyp+fQPX+S/P3Ul9VlEKJHU/PHduxn2R1DKuO+0wXLuHfQD8NCeE0UJ/dHRIF9/vJd7XziO06F4w1md/PTVQUYDkbLnVAinpoxjPhmab/XOZWgqwupmL3f9+ig7ekf48ru2ctH61rLev28owFkrG/mfT19Z0Pb+cIxzv/AzTk6G8m57ZGQagKf+7GrWtvlmPXb1l5+gb8hf/IRtoKCoG6WUG0Pkf6C1vs8cHlRKrTQfXwkMmeMDQHfa09cAJ+yZrlCNHB83fgD9Y9NFP3ckEAUM8fuHR/bbOq9cHBoJEE9qpsJxRqejRT8/kdQMjAe5eH0rx8aCfOHBV7Nu+43H+9jRN8LW7ma0holg8e+XiQODAQBeOT7FoeFA3u0nglH+/J49XPOVJ/jJ7hP87iXrePozV/OXbz0bgHGb5pWPoSnjhDKVwb2RTjiWIBCJ876L13L3rZcC8J7bn+H/PbyXcCxR8vsfHJ7mjI66grdvqHVTX+Pi5GQ477ZHRqdxORSrmue7Z87oqKd3MP9xqgSFRN0o4A5gr9b6q2kPPQjcYt6+BXggbfwDZvTNJcCk+OeXL+FYghHTEjwyGiz6+cP+CPU1Lj585Xrue/E4Lx2bsHuKGdl/asayOjxS/AnqxESIWEJz0/mr+cTVG7ln1wD/9dJ8e+bZQ6P8488PcOO2VXz4ivUAjJVwYslE75Cf1c1elIKH9uT/if39I/u594UBfscU+C+87RxWNNbSWucx55XfwraDQdOinwrFc2437De+Vx31NVy0vpX/+fSV3HzRWm5/6hA3/MsODgwWbx2HYwmOjQc5o6O+qOd1NdVyqgCh7x8L0t3qw+WcL62bVtRzeGSaWCJZ1HvbQSEW/eXA7wLXKKV2m3/XA18E3qiU6gXeaN4HeBg4BPQB3wY+Zv+0hWphYHzmcra/BMEcCURor/fwsavOoL3ew/996DdFLZKVyr40oS/EGp7LsTHjpLau1cenrt3E+Wub+fz9LzMwPnOyGwlE+PQPX6SnrY6/ffu5tJmCal3FlEvvYICLN7Ty2p5WHnzpRM7PbSQQ4d4XBnjX9u6UwFv4PC5q3Q7GphfGdTPkL8yitwyI9gbjc6urcfH/3n4u3/291zLkj/ClR/YV/d6HR6bRGjZ2Fin0jbUFWfT9o9OsbfVlfGxTZz3xpKa/BIOoXPIKvdZ6h9Zaaa3P01pvM/8e1lqPaq2v1VpvMv+PmdtrrfXHtdZnaK3P1VpLWcplTLqwlWLRG0JfQ0Otm//1pi3s7B/nv1+u/AXg/lN+NnbW43E6ODRc/Amq3xT6tW2G9fbP7zkfreGP795NPJEkafrlx4Mxvv6+C6ivcdFWb1nO5Qv9VDjGqakwmzobuGHrKvqGAuzPYeF+/5l+ovEkH7lyfcbHW32eBbPoh0yL3h/ObdFbJ8T2+ppZ41dt6eS8NU0Ml3DCPGie1Cth0Wut6R8J0tOWTegbABbFTy+ZsUJZWBZ9T5uP/tHiBXPYH0n9kN+1vZuzVjbydw/vK8sHWwj7Tk5x9spG1rX5OFTClUj/aBC3U6UWj9e2+fi/N53D80fG+cYTB7ntyYM83TvCX91wNmevagRIc5GUbzlbvt7NK+p5y2u6cDpURtcRQCia4N+fOcIbzlqRVeBa6jwL56O3LPo8i7Epi36O0AM0ed15n5+Jg0PTKAXr2wv30QOsbKplyB8mnsPtMh6M4Y/EWduW+bXP6DTGF8NPL0IvlMXAeAi3U3HR+tYSLfooHQ3GD9npUPzlW8/i+ESIO3YctnuqKSZDMU5MhtnS1cCGjrqSXDdHx6ZZ0+KbFT3z9vPXcNO2VfzzY7189dEDvPW8lbzvopnQ4RafIfSlLP7OxYq42dTZQHt9DZed0cZ/vXQyo/vmnhcGGA/GuPV1G7K+Xmudx7a1g3wMpiz63EJt+eitK6F0mrzugqJ25nJwOMDqZi9eT3E5G11NtSQ1DOeITDpiGjrZLHqfx8XqZi+9QyL0whJjYDzI6mYv69vrGQlECERyX46nE40nmQzFZllsl53RzhvPXsE3Hu9LhV7ajbWId2ZXAxs66jk6FsxpqWXi6Fgwoy/2/9z0GlY3e+lu8fJ37zh3Vtim2+mgyeu2RVB7hwLUuh2saTGuKG7YuoqjY0H2DEzO2i6R1Nzx9CG2djfz2p7sMfItvkWw6PO6biI01rqocc0XZUvoi13P6RsKFO2fB8OiB3K6b46ahs66LEIPxoJsnwi9sNQYGA+xpsWXsmKKcd+MTs9ebLP4/PVnEU0k+cpPD9g30TSshdgtXQ1saK8jltAcG88fI22htbGglukH3Vjr5qFPXcFDn7qShtr5OQVt9R5GbViMPTBorDFYmazXndOF2znfffPobwY5Mhrk1is35MwVaK3zMGbTInEuwrEEk6EYSuW36EcCkdTV3lyavG4SSc10tHAXXzKpOTQSKNo/D9DVaJxQcwn9kVHDLbSmJYfQd9ZzcDhAIrmwqUMi9EJZHJ8IsabFyzrTL3lkpHD3zYg/82Lb+vY63n/xOn606xiTQfsXCPefmqKhxriM3mDGUx8eKdzKmgjG8IfjWaMrGs2460y01XlSJ7hy6BsKsNlc3AND+F6/uZOH9pwkmSYi3376EN2tXq47J3fZhdY6D/5InGi8sqF/ljumu8VHOJbM+X4j/mhG/zyQKp1QjPvmxGSIcCxZktBbFn2uyJv+0SCrmrw5S3ls7KwnEk/OCmJYCETohZIJxxIM+yOm0Buid6QIiz7XYtuF61rQGgYr4L7Zf8rP5q4GlFJsaDd+9MVE3qQibrIIfS7s8IVPhWOcnAyzccVswbph60pOTYXZ2T8OwK7+MXb1j/Phy9dnjOtOp8VcKLYrmSsbln/ecp/ksupHAhHac1j0QFGGgOUyKSZZyqLZ56bG5Uhl9WYiV2ilxUbz5LzQC7Ii9ELJHJ8w3B2rW7zU1bjoaKgpynVjWXedGX7MVsy5HW6OdLTW7DvlZ0uX8YNrqfPQ4nNzsAihP2rF0GeJrshFa11N2UJvCVa6RQ/whrNWUOt2pNw3337qME1eN+/a3j3vNebNy1woHquw0Fv+eUvoc/nph/0ROmy06K1jXIqPXinFyqbcsfT9o0F62vMJvfHefSUEAJSDCL1QMlZopeWT7GnzFRV5M5zDom8zx+xwc6RzcjKMPxznzK4ZkdzQUV9U5M1R82RWikXfZlr0yTJ8tKmImzkWfV2Ni2vPWsHDL5+kbyjAT39zit+5ZG1BhcBa6gzhrHTkTcqi78ht0YdjCfyROO0ZIm4AGksS+gDNPncqzLVYjFj6zGs5/nCM0ekoa1tzn/ybvG5WNNaIRS8sHSw/oxX5sa6triiLfiQQoc7jzBjqNhNzbq/wWKUPtqyYEfr17XVFlUHoHw3S2VBTdIgeGIuxSQ0TJYQGWvQOWhE38080N5y3itHpKJ+860XcDge3XNpT0Gtan/d4hZOmhvwR3E6VKviVrQyC5dbLtRgL+bNr0zk4ZCzEllLADmBlkzerRW9lu2YLrUxnY2f9gidNidALJWPF0Hc2GAtVPW0+BqciBKOFhViOBKJZfbAtPjdK2e+6sSJuzuxqTI1t6KhjyB/JGwVi0Z8ltLIQ7EiaOmAKVqYKmFdt6aC+xsXek1PcuG0VnQXWPrczmSsXg1NhOuprUkKd7TPPlhVrYVVJLSZp6uBwoCT/vEVXUy2DU+GMV2OW0M+tWJmJTZ0N9A0FFqTUh4UI/SLxrm/+ijsrmBS0EAyMh1jV7E0JjuWztnzY+RhJy4qdi8vpoNnrtt11s//UFF2NtbPKKVsLsoVa9UdHgwX9oDPRVme6pMo4gfUN+tm8oiHjY7VuJ28yI2x+P0eC1FysZK5Kl0EY9kfobKxNuV6yWeQj/uxuPYB6jwuHKtx1MxGMMhKIluSft1jZVEssoTMmvFmVWwtZt9nYWc90NFFQ7Ry7EKFfBOKJJDv7x3lxgSo1VoqB8WDKbQPQU2SI5XAg+2IbVCZbM30h1mImxDK/0IdjCU5NhVmXxxebjXJdUv6wkdWbS7A+c92ZfOcD27OeDDLhdjpoqHVVPGlqcCpMZ0MNDbXGukE+1022Kz6HQ9FYRHastRBbSmilRVdj9qSp/pEg7fU1WcNq09lkHruFzJAVoV8ERgJRtJ6xWpYqA+Mh1jTPWLZri0yaMsLnsi+MtdXX2FbpESCWSHJwODBrIRaMTEaHoqDIG2tdIlf2Yy6sxcWREoU+FXGTQ8S7mmp5w9nFtytciDIIQ/4IKxprqfe4ciZNpcof5Fg4LaYMQqnFzNKx6hplakDSPzZd8HfCOkn3llBmuVRE6BcBK/IgV92Maic9ht6iyWtENBQSeRNLJJkIxrJemsNMhIpdGLXA9TyLvsblZE2Lr6DIm2J8sZmw4tVLzUK1ojU2leGCyEalyyCEYwkmgjE6G2pwOBT1Na6s4ZVW+YNcyUdFCf1QAI/TMev7WiwrmozvaqZY+myZ0ploq6+htc6TOvksBCL0i4AVSzy8hC36E2YM/ZrW2T+cnjZfqp1aLkbzLLaBVS7Avs8ovfTBXIziZvnnnRL6Ehdj3U4HjbWukhc9e4f81LgcdJf4/rmotEVvfd+tWviNte7sPvocC/UWjbXFWfTr2+vyJo7lor2uBpdDzfOth2OGv70Yd97GzoXtNiVCvwgMpvXMjMQrW463Ulgx9KubZwtOT4EhlrmyYi1a62qYCMVsqwuy/9QUTofK6N+2QizzRUIcHQtS53HmdCnko62+puQKlgcGs0fclEulhd4qUtfRaBzzRq87q49+OJB9od6iONfNdKpMcKk4HIoVjfPr0ltNaPIlS6WzsbOe3gWMvCmkleCdSqkhpdQraWN3p3WbOqKU2m2O9yilQmmPfbOSk1+qDKVd+tkdPrhQzCRLzbbo17XVcWIynLeefKpNXA6rrb3eg9b29TLdf8rP+va6jNUQN3TUEzIXWnNxdCzI2ra6kmOxwax3U+Jx7xsKsHmF/W4bmBH6SonPoNkrdoUZjttQ68oRXpl7oR6sE0V+oY/EExwdK759YCaM7NjZPvojJVzlbeqsZzIUWzD3bSEW/XeBN6cPaK3fY3Wbwmgafl/awwfTOlF91L6pLh+G0lw2S9V9MzAexGVaOOlYVs2xPCGW1hc8X9QN2HcyzBRxY3GG2Ygin/umf3SadWW6TUq1nAOROMcnQmwqIpqmGFp8HiLxJKEKNX2xDJxOy6KvdWf10RsNaXJfNRVaqrh/NEgiqcsKrbTI1GmqP1WHvvArhpluUwvjvimkleBTwFimx8zG4e8G7rJ5Xsuawalw6tJ7ZIkuyM6NobdIVbHMsyA7tx9oJlJCb0MsfSASZ2A8xJlZRHJ9hyX02X94yaRRzrjUiBuLtnpPSa4bSxQqsRAL0FrhMgiD/gguh0rV1WnMYtGHYwn84XhBrptYQuc9MR0cKj/ixsKqd5N+cukfDdJY66LZN78sdTas8hVVI/R5uBIY1Fr3po2tV0q9qJR6Uil1ZZmvvywZ8kdSP9ZKWPT/9PMD/GLfoO2vm87cGHqLQuvSj/ij+DxOfJ7sccfWD90Oi35/joVYMGKkfR5nzraCp6bCROPJshdCW822fcXWuzmQqnFTOYseKlcGYWjKqC9v1dDP5nqxToK53HpQeGEzK7ql2PaBmehq8hIxG+ZYHBmdZl2R7rzOhhoaalwLtiBbrtDfzGxr/iSwVmt9PvAnwH8qpRozPVEpdatSaqdSaufw8HCZ01haDE5FUn1EKyH0dzx9mB/vHLD9ddMxGo7MF/pmn4cmrztvueKRAhbb7Kx3sz9D6YN0lFKsb88deTNTtbJMi76uhkRSF90Kr28ogMflKDniJ++86u27gsrEkD88qyRDQ62LQCQ+74SXLyvWolCh7xsKsKqptqDibvnIVJf+6FjhoZUWSik2rqind4Fq3pQs9EopF/AO4G5rTGsd0VqPmrd3AQeBzZmer7W+XWu9XWu9vaOjo9RpLDniiSSj0xHWtPho8rptd93EEkn8kTjHKtjYIBJPMOSPZO2kYzQKz++6yWextfg8Zr2b8j+j/aem8HmcOeOoN3TUcyhHA5JUq7gSs2ItZgS1uBPYgUF/xSJuIM2ir1As/dBUZFZJ6sZaN0kN03NqI+XLirUotCa9EXFjj7ura05LwVgiyUCJ7rxNnfX0DRXfmL4UyrHo3wDs01qnTEelVIdSymne3gBsAg6VN8XlhZUV29lQQ0dDje2r7hPml/7YWOGt8YrlxITxJc8mmuva6vJa9IUstjkdihZfaf7suew7ZdSHceQQyfXtdQyMh7KGvPaPTeN0KFY1F1YoLBulXqn0DlYu4gbS51UZ182gP8yKxhnxtsog+MNZhL6AxVjIbdFrrc1iZvZ8bnMt+hMTIRJJXVJvgk2dDYwEIowvQFP2QsIr7wKeAbYopQaUUh82H3ov8xdhXwfsUUq9BNwDfFRrnXEh93TFiqFf0VhLe73HdtfNZChq/o8VVcK1GGbKE2e36I+Ph3K3iSvAdQP2xHZrrdk/6J9X+mAuZ3TUoTVZr0b6R41G6OUk3UBplSJTETcVWogFw8J2KCoiPJG4lRU7c5LMVths2EbXzampMMFowjaLvqO+BociVZf+SOoqr3iLfiGbkOR1Wmmtb84y/sEMY/dihFsKWRhKZQfW0NFQyyvHJ219/fG0y9hjY0HOWdVk6+tDWrJUDos+qeHYeObY5VgiyXie8gcW5cScWwz5I0wEY1kXYi1m2goGMtaSOVaCLzYTqUXmIgQ1FXFToYVYMBKCWnyeinSZGpqa+d5bZCtsNhKI0pCn/AGk16TPXha7nPaBmXA5HXQ2zHSaSoVWlrDQO1PzJsBre1ptmV82JDN2gbEs+s6GWjrqa2y36CdmCX1l3DepGPosPlQrlj5b5I1loefzwYIViljeZ5Sr9EE61ryzFTcrpw59OpYvvJgTWKqrVAUtejBq8VTCoh9KtY1Ms+hrM9ekz1fV1KKh1iiMlsuit0Ir7Yiht+hqqk0l1vWPBql1OzK2w8zH6mYvXrdzQUIsRegXmKGpMEoZ/sf2Bg+BSJxQ1L4ElfSFtEp1mh8YD7GyuTarCyNfueJUVmxBFn3p5QIs9p+aArJH3Fg01LrpbKjJWK54MhRjIhizxaL3uIySwMW4pHorHHFj0Vpnz5rIXOYmS0GaRT9H6HP1KUjH4VA01LhyZsceHJ6modZV0HetUNJ7xxoJdKVlSjscirNXNVYsQS2d8uONhKIYMr/ELqcj9eUbCURsK1I1YQq9x+nIm51aKnPLE8+ltc5DQ40rq0WfyorNkSyV/loTwRjxRLJk3/i+U346GmoK6hVqFDebb2EdTaW52+MCaCtSUHsH/WwosyhXIbT6KlNVMaNFn+oyNX8xNt/Vl0WTL3e9G2shtpySFXPpaqrl6d4RwLDoy4nPv+ejl9o6t2yIRb/AWI0XYMZ1MWSj+2YiGMPlUGzoqEv50u0mW7KUhVKKde3ZG4UXGidtbGOF/JW+sLz/VP6FWItzVzfxyvGpeS41K4beLovaWGQu/LgfGMy8bmA3LXWVKVVsZYOnF4Ob8dHPsegD0YK+G5C/sFnfkH0RNxYrm2oJROJMhmL0l7lusxAiDyL0C47VeAFmXBd2+unHgzGafR66W30ViaWPxBMMTmWPobfI1Sg8Xz/QdFqt1nsl+unHpqPsP+XnvDWFLUq/57VriSaS3P380VnjVqu4UuvQz6WtvqZgH/30AkTcWLTWuRkPxorO2s3HkD+SqkNvUeNyUuNyzLLoI/EEk6HCFuoht9BPhWMM+SO2+udhpszy7mMTROPJkkIrFxoR+gVmMC1pxPpvZ9LUZChKs89Nd4uPY2Mh2ysRnswTQ2+xrtXHsfEQ8cT8EMuRQASv21lQpmJrmY06/nvPCeJJzVvPW1XQ9hs767lyUzv/8exRYmlzPzoapL3eU1CruEIoxnWzEBE3Fi0+D4mknudOKZf0K9l0GubUpLdOfvmS6SxyCf2hVPtAe4XY6jT13KFRoLhiZouFCP0CYmXFWmngrXVG5qetFv10jBafmzUtXkKxhO0La9nKE8+lu9VHIqkzNkDO10IwnXJb7/1k9wm2rGjgrJW5F2LTueXSHk5NhXn0NzP1gvpH7Ym4sWg1o1sKORE/ecAoEXLOqsL3oVRSJ1ab3TdWU/C5NHpnd5kqpE9BOrmEPlXMzGaL3kqaetYUejsW6CuNCP0CYmXFWrHELqeDVp/H1uzY8WA05bqB/OWCiyWVLJVH9LpN104m99Gwv7DwOUi36Iv/jI6NBdnVP87bthVmzVtcfWYna1q8fPdXR1JjRj0T+yy31joP8aTO2njDIhJP8P1n+rlqS0dFukplmhcUl8xVCNks+sba2YXNCs2KTX9+NqHvGw7gcijbI5WsyKE9A5O4nSol/NWMCP0Ckh5Db9HRUGNrk/DJUIxmr5tus8XfMZsXZAfGQzhzxNBbWO8/kCGWv9CsWDCKpDlU8XVhAB586QQANxYp9E6H4gOXruPXh8fYe3KKaDzJicmQrUI7kzSV+9g/sPsEI4EIH7lig23vnYtKlEGIxBOMB2PzeheA1XwkzaL3F75+A0bkTjSezNjopncwQE97HW6bI5VqXE7a640T9ZoWX8Ujoeyg+me4jEjPirVor7e33s14MEpLnWfGoq6ARb+yKXsMvcWqZi8OldmiL6QfqEWp9W601vzkxeO8tqcl78JxJt69vZtat4PvP3OEgfEgWpeW5p6NmVr72fdLa80dTx/mzK4GLt/YZtt752KmVLF9rpvhVGhlBoveO9tHPxN6W7jrBjInTe0fnCo42qpYrOJmS8FtAyL0C0pWi94moQ/HEoRjSZq8bupqXLTWeWxPmspWnngubqeDlU3eeSeaeCLJeLDw8DkwQxGLXIzde9JP71CAt21bXdTzLJp9Hm7atpr7XzzOy2aZCjt/1IV0z9rRN8L+QT8fvmL9goXhVcJHP2PgZPDRz7Hoh/0RGmrylz+wyCb0/nCMY2OhotZmiqGr0fgN2HnyryQi9AtIelashVXYzI7oGKv8gWWVdbd4bS+DYAh9YV/u7lbvPNeR0ZMUOgr0wUJpZRAe2H0cl0PxW+euLOp56Xzg0h7CsST//HOjr45doZUwU6o4V3bsd54+THt9TdFrDOXg8zjxuBy2WvRWVmwmKz2Tj77Qqz3ILvRWk5ZKWfQrUxZ99UfcgAj9gpKeFWvR0VBDOJYkECk/nM1KdGkxW5qtsTmWPhJPMOgPF2TRA2aI5+z3t6y7Qi/NofgyCMmk5sGXTvD6zR0FZcNm4+xVjVzU08qhkWm8bqetafT5Fj0PDPp58sAwt1y6LmMz80qhlCo6azcfuSz6hloXkXgyVRraWL8p/Jhlq0m/96Qp9JWy6E2ht+ojVTsi9AtIpsiDjlQsffk/LMuibzKFvrvFl6qXbQcnJ8Jonb088Vy6W30M+SOzFsqKDZ8D06Iv4vP59ZExTk6GbbGEP3DZOsDIiLXTfVLjctJQ48p63O/ccZgal4P3X7LOtvcslBafvYXNMmXFWswtg1BMVixkt+j3nZqiodbFqgpFxJzRUY9DzTT5rnZE6BeQ9KxYi3Ybs2MnUha96bpp9RJL6FSlvXI5PlFYDL1FKvImzX1TTFasRWudh8lQbFYCUy4e2H0cn8fJG88mHsFxAAAgAElEQVReUfB7ZOO6c7pY1VSbauZsJ631mWvtjwQi3PficX77wjVlXZGUPK86e0sVD00Z4bSZmr7MLYNQTEQWpJcqniP0J42yF5Va23jT2St47E+vWpCQVzsQoV9ABue0UoMZi94WoTd/LM1pFj3AgE2RNzMNRwp33cDsyJtC28Sl02b+8AuxMiPxBP+95yTXndOVs/F4obidDu792GX8nxtfU/ZrzSVbU5X/eLafaDzJhy5fb/t7FoLdpYoH/ZFZVSvTmSlVHCcaTzIRjBXl1mvMYNFrrdl3yp+3Wmk5OBzKlmbjC4UI/QIxNyvWIr2CZbmMz7PoLaG1Z0HWiqHvyuBrzYT1/uknmhF/hFq3gzpP4X7ntgJCES2e2D/MVDhu6wLmyiZvRSzrTGsP4ViCf3+mn2vO7LS9RkuhtPrctjRktxiaCs+KNEunoXbGIrcW3Iux6J1mqeJ0oR8YDxGIxDlz5dJwqywEhbQSvFMpNaSUeiVt7AtKqeNKqd3m3/Vpj31OKdWnlNqvlLquUhNfaszNirVo8XlwOpRNrpsYtW5HKjRtVXMtStkXSz8wHqKrMX8MvUVHfQ0el2PWicZqCl7MJXUxPVYf3H2CtjoPV25sL/j1Fwuje9bs4/6dpw8xOh3lI1csjjUPhkU/FY4X7CrLx1Aui9470zd2JlmquJNq45wyCFajmUpa9EuNQn6x3wXenGH8H7XW28y/hwGUUmdj9JI9x3zON6xm4ac7mWLowbgEbKuzp3fsRDBKs3fmR1LjctLVWGtb5M2R0emC3TZg7Nualtmx9MNF+mAhrd5NnquekUCEn+8d5LfOW7kkshVb642SwFZo7TMHR/nqowd463krufSMhUmQyoR1BWVHueJoPMnYdJQV+Sz6UKwktx4Yfvr0EM19J41GM4XWtD8dyPtr0Fo/BRTa4PtG4Ida64jW+jDQB1xUxvyWDZmyYi3sSpoyShS7Z42tafFmLENQLIFInJcHJrlgXUtRz+tumR3iOeIvLqoCZkoV57Pob3/qELFEklsu6ynq9ReLtjoPsYRmKhxnyB/mk3e9SE97HV/87fMWLEEqEy2W0NtQBsHKdM3uo5+x6FNZsUV+P+YWNts36Gdtq8+2SqPLgXLMnk8opfaYrh3r178aOJa2zYA5dtpjWfSZYontKoMwmUHo5wptNv7+kX389m2/yvr4rw+PEk/qol0i3a2zk7aKjaoAaPa6jXo3OUIsh/xhvv/MEW7attr2RhOVwkqaGvZH+NRdLxKIxLjt/RcuukC1+gp3leVjKPW9z3zM6zxG39epcKyk0FvIIPQnK1f6YKlSqtDfBpwBbANOAl8xxzOZIRmDuJVStyqldiqldg4PD5c4jaWDlRWbKZa4o8GeJuHjwWhqIdZiTauPU1PhVEJKxudNR/m3Xx5mV/94Vn/+070j1LgcRVv0a1t9TIZiTIWNdoBjwWhRWbFguIDy9TL91pOHiCU0n7x2U1GvvZhYVyp//V+v8uyhMf7mpnOrwt3QYqPrZnBqfgvBdKy+r/5wnGF/hPoaF94iFuphttCHYwkOj0xXLFFqqVKS0GutB7XWCa11Evg2M+6ZAaA7bdM1wIksr3G71nq71np7R0dHKdNYUmTKirVorzdcN+WWQbC6S6XT3eJFazgxkT2W/gfP9ROOGQtvVi/Mufyyb4SL1rcWXINk5v1niquNBc3yB0X6YMFsWp3lqmdoKsx/PNvPTdtWL6mQN+uk/3TvCO/Z3s07L1yzyDMyKGbxOx/D/vlNwefSYJZBMJKlio9uSu8b2zsYIKnhrCo4YVYTJQm9Uiq9gMjbASsi50HgvUqpGqXUemAT8Ovyprg8yFaPGwzhiyV0zt6X+dBap7pLpZOvLn0knuB7z/Tzus0ddDXWsqNv/tXV0FSYA4MBrighkmXm/UOpq5ZiL83BCEXMJjzfeOIg8aTmU9duLPp1FxPLdXPWykb++sZzFnk2M1jfITti6QenImZWbPZjblSwjDPiL96tB4ZFH44ZZRT2njIWYsWin01eZ6BS6i7gKqBdKTUA/BVwlVJqG4Zb5gjwBwBa61eVUj8CfgPEgY9rrbP7DE4jMmXFWqQnTc21yAtlOpogltCpOjcWM7H0mYX+gd0nGPZH+Oq71/PA7hM8+ptBEkmNMy2LcUefYeVfXorQW0lb48HUJXmxURVgRKjsPTE1b/zUZJj//PVRfvuC1UumwJRFV2MtX7jhbN54TlfRV0qVxCrPcHwixI7eEXb1j7Oz3ygr8a/vu6Ao99KQP0x7vWfW92kuDbUupsIxxqejJa2vpCdN7Tvpx+t22t5sZKmTV+i11jdnGL4jx/Z/C/xtOZNajgxORbI2qLYuV4cDkZL7glrlD9LDK8EQE7dTZaximV7v/IqN7YxNR7ln1wCvHJ9ka3dzarsdfSO01nk4uwQrqcnnpqHWxbGxYGr9oBSrrb3OkzEy6bYn+kgmNZ+8Zun45i2UUnxwkbJf89FS5+GHzx/jh88fQynYsqKB8ekoH/7e8/zk45cXfAwHp7IbOBaNtW4GxoOMBCJcvKG16Lk2ppVR2Hdqis1dDTlPLKcjEn+0AFhZsR1ZFqQ6bSiDYBU0m+u6cToUq5q9GS36p3uNeudfftdWlFIpi31H30hK6LXW7Ogd4bIz2jLWKimE7hYfR8eCrGo2YvBL8cO21tUwZabJe1yGx/HERIi7fn2Md164ZsnUHFkq/Nl1W+gd9HNhTyvnr22msdbNS8cmePe3nuEP/n0XP/jIxbOuQiLxBP/08156BwN8+V3npa5Mh/wRVjfnEXqvi/HjUcaDMTrqiy9Cll7YbO/JKa47p6vo11juVH9WyTIgW1ashfXlLkforQiJTK6f7hbfrMJiFt9++hAdDTXcsNVYcmmvr+GslY3sSFuQ7RsKMOSPlOSfT72/WZd+JGCUPyglfNDyZ1v7GYjE+d8PvEpSaz5+9dLyzS8Fbti6ij950xZev7kjVY9ma3czX333Nnb1j/O5+15OBQ/sPTnFjV//Jbc9cZAn9g/x3tufTV19DU2FMzYFT6ex1p2Kzim0aXw6ltD3DgYYD8aqInKp2hChXwCyZcVaNHpdeJyOskoVzzQdcc97rLvVy8GhAL/sG0n9OPedmuLp3hE+eFnPrHrnV25qZ1f/OKGosbRi+eev2FSG0Lf4GBgPpiKPSkkGakvryLSrf5zr//lpHts3yGfevEWs+QXkt85byf9602buf/E4//KLPm5/6iA3fv2XjASi/NsHX8u//d5rOTI6zbu/9QzHxoKMTkezBiFYWK4XKM2tZwn9c4eNvE4pfTAfcd0sALmyYsHw1VqdpkplIodF/+7t3fxi3xDv/85zbO1u5hNXb+SRV07hdTt5/8VrZ217+cZ2bn/qEM8dHuWqLZ3s6B2hp81XUt9Vi+5WH+FYkv2n/CX9kGEm5O8rP9vP4/uHWNnk5e5bL+Wi9cX7dIXy+PjVGzk4PM1XHz0AGCV7/+4d56aqjH7/Qxfzoe8+z9u/YSTg5fPRW2UQoDyh/3VK6MWin4sI/QKQKyvWor3MMgippiPe+Rb9+WtbeOozV3PvruN888mD/P73dwLwgUvXzTsxXNTTisfpYEfvCJdvbOfZQ6O8/YLykputuvT7B/1ce2ZpNeItEXls3xC/fcEavvC2s2cJhLBwKKX4u3eci8/jZFt3M++8cM2sq7SL1rfyg49czAfuNCKr81r03hkZKqWLlxV1c3zCKLrXsgg1/KsdEfoFIFdWrEVHfQ0nJktvEDIejFFf40otVM6lxuXkfRev5d3b1/DQnpM88sopPvr6M+Zt5/U42d7Two6+Ea47NsF0NFGWfx5mQiyNZKnSfoTr2nzcfFE3V27q4Poy+sAK9lDrdvK3bz836+Nbu5v54a2X8LXHejl/be5s6lkWfQnfD7fTKHs9HU1IaeIsiNAvALmyYi1Wt3h57vAYWuuSfNgTwWhGa34uLqeDm85fzU3nZ7fSr9jUzpce2c/9Lx7HoeDSDeUJfbrbp9S+q26ng797x3llzUNYWM5a2chtv3Nh3u2sxd46j7PkZjFNXrch9OKfz4gsxi4AubJiLc7saiQQiWeMjimEiVCMljp7XBlXbjRKUtz9/DHOXdOc6kFbKl6PM5UUVkqylLC8sdoJlvPdsNw3Z4lFnxER+gUgV1ashRUSZjVNKJZMBc1K5ZxVjbT43CSSmis22lMXvbvFiqEXoRdmY4l0Od8N62pWLPrMiNAvAEZ2YO4vsSX0+0/NT/MvhMlgrCDXTSE4HIrLTL/8FRvtKThnhUCK0AtzSVn0JSTSWTR53bidig0dS6sMxkIhPvoKky8r1qK+xkV3q5e9VWDRA7zrwjUMT0W4YF1z/o0LwFqQLefHLCxPZoS+dCPgys0dtPg8uJdAZ7HFQIS+wuTLik3nzK5G9pcg9MmkUflybvmDcrhqSydXbem07fWu2NTOjr6RVBkEQbCocTl53eYOLjuj9EX/371knY0zWn6I0FeYVAx9HosejBrav9g3RDiWKKqaoT8cJ6kzJ0tVC5dsaOMnH798sachVCnf/5B0HK0kcp1TYays2FyNFyy2dDWSSGr6hgJFvYdV/yVT+QNBEAQR+gpTSFashZXsUWzkzUQoc+VKQRAEEKGvOIVkxVr0tNVR43IUHXmTq3KlIAiCCH2FGZ2O0urz5MyKtXA6FJtXNBRv0aeajohFLwjCfPKqj1LqTqXUkFLqlbSxf1BK7VNK7VFK3a+UajbHe5RSIaXUbvPvm5Wc/FIgGE3gqyl8YXVLVylCb5UoFoteEIT5FGLRfxd485yxR4HXaK3PAw4An0t77KDWepv591F7prl0CUbj+NyFBzed2dXAsD/CaBGVLMeDMZSayTAUBEFIJ6/Qa62fAsbmjP1Max037z4LrKnA3JYFwWgi1RS7EKwU7mLi6SeDURpr3dInUxCEjNjho/8Q8D9p99crpV5USj2plLoy25OUUrcqpXYqpXYODw/bMI3qJBRN4CtG6M3Im2IyZMeDMQmtFAQhK2UJvVLqL4A48ANz6CSwVmt9PvAnwH8qpTJWGdJa36613q613t7RYU89lWokWKTQt9fX0F7vKSryZjwYpUn884IgZKFkoVdK3QK8FXi/NhuRaq0jWutR8/Yu4CCw2Y6JLlVCsQTeImtsn9nVWNSC7GRILHpBELJTktArpd4M/DnwNq11MG28QynlNG9vADYBh+yY6FLFWIwt3KIHI/LmwKCfRFIXtL3dBc0EQVheFBJeeRfwDLBFKTWglPow8HWgAXh0Thjl64A9SqmXgHuAj2qtxzK+8GlCsYuxYETehGNJ+kenC9p+wsYSxYIgLD/y+hS01jdnGL4jy7b3AveWO6nlRLGLsTA78mZDR33ObeOJJP5wXCx6QRCyIpmxFSQaTxJP6qKFftOKehyqsJo3UudGEIR8iNBXkFA0AVD0Ymyt28mqZi9HCnDdWFmxIvSCIGRDhL6CBGNGTlmxFj0YBc76R4N5t5tIlSgW140gCJkRoa8gQdOiL0Xo17b5ClqMFYteEIR8iNBXkJTrpsjwSoCeNh/jwRiTppBnY1wsekEQ8iBCX0FmLPriOzauazO62fePzbbqf/bqKX7ra08TjhmvbVn0TWLRC4KQBRH6ChKMGj76YuPowfDRA/P89I+8eopXT0zx/BEjPWEiFMXlUDTUSPtfQRAyI0JfQULl+OhbfQDz/PS7j00A8OR+oxDceDBGs8+NUlK5UhCEzIjQV5ByFmO9HicrGms4kmbRT4ZiHBo2hP/JA4bQT0pWrCAIeRChryDBmBVHX7zQg+GnP5om9HsGDGv+9Zs76B0KcGIiJHVuBEHIiwh9BQlFrTj60vznPW2+WUlTL5lum09esxGApw4Mp1w3giAI2RChryDBMsIrwbDoh/yR1KLu7mMTnNFRx4XrWuhqrOXJA8NMBqM0i0UvCEIOROgrSCiawONylNzib12btSAbRGvN7mOTbO1uRinF6zd3sKNvhLFglGbx0QuCkAMR+gpSbHepuaSHWB6fCDESiHB+dzMAr9/SgT8cJxxL0lInFr0gCNmR4OsKEowmim46ks7atpkQy3gyCcBWU+gv39iO06FIJLX46AVByIlY9BUkFIuXHHED0FjrprXOw5HRIC8dm8DjcqRq1Td53WwzRb/ZKxa9IAjZEaGvIEbTkfIumta1+Tg6Ns3uYxOcs6oRj2vmkL1+s9FUXfrFCoKQi4KEXil1p1JqSCn1StpYq1LqUaVUr/m/xRxXSqmvKaX6lFJ7lFIXVGry1U4pbQTn0tNWx8GhaV4+Ppmy4C1u2raaSze0cdbKxrLeQxCE5U2hFv13gTfPGfss8JjWehPwmHkf4C0YTcE3AbcCt5U/zaVJKFbeYiwYpRBOTYUJx5LzhH5tm4+7br1EFmMFQchJQUKvtX4KmNvk+0bge+bt7wE3pY1/Xxs8CzQrpVbaMdmlRrlRNwA97b7U7blCLwiCUAjl+OhXaK1PApj/O83x1cCxtO0GzLFZKKVuVUrtVErtHB4eLmMa1UsomsDrLtdHb4RYNvvcqUJngiAIxVCJxdhM2UF63oDWt2utt2utt3d0dFRgGotPMBov36I3hX7rmmapUCkIQkmUI/SDlkvG/D9kjg8A3WnbrQFOlPE+SxY7XDctPjdbu5t582u6bJqVIAinG+UI/YPALebtW4AH0sY/YEbfXAJMWi6e04lEUhOJJ8uOulFK8cDHL+fmi9baNDNBEE43CnIgK6XuAq4C2pVSA8BfAV8EfqSU+jBwFHiXufnDwPVAHxAEfs/mOS8JQrHSa9ELgiDYSUFCr7W+OctD12bYVgMfL2dSy4GZNoJSZUIQhMVFMmMrRKqNYBm1bgRBEOxAhL5ClNNGUBAEwU5E6CtEqumICL0gCIuMCH2FSLluxEcvCMIiI0JfIYKpfrFi0QuCsLiI0FcIK7xSXDeCICw2IvQVQhZjBUGoFkToK0RK6MssaiYIglAuIvQVIpRKmBKLXhCExUWEvkIEowlcDjWr9Z8gCMJiICpUIexoIygIgmAHIvQVImRDiWJBEAQ7EKGvEMFYQpKlBEGoCkToK0QoGscrBc0EQagCROgrhB3dpQRBEOxAhL5CyGKsIAjVQslOZKXUFuDutKENwP8GmoHfB4bN8c9rrR8ueYZLlFA0QWdDzWJPQxAEoXSh11rvB7YBKKWcwHHgfozWgf+otf6yLTNcogRjcXHdCIJQFdjlurkWOKi17rfp9ZY8oWhC2ggKglAV2CX07wXuSrv/CaXUHqXUnUqpFpveY0khi7GCIFQLZQu9UsoDvA34sTl0G3AGhlvnJPCVLM+7VSm1Uym1c3h4ONMmS4YXj47zF/e/jNEXHbTWhGIi9IIgVAd2WPRvAV7QWg8CaK0HtdYJrXUS+DZwUaYnaa1v11pv11pv7+josGEai8fj+4f5wXNHmQjGAAjHkmgtBc0EQagO7BD6m0lz2yilVqY99nbgFRveo+roH53mCw++SjKpU5UqhwMRYKbpiE8SpgRBqALKEnqllA94I3Bf2vCXlFIvK6X2AFcDf1zOe1QrP3t1kO/+6ghD/khK2Ef8htDPtBGUxVhBEBafspRIax0E2uaM/W5ZM1oijJjWe0LrVJORlEUflTaCgiBUD5IZWyKWqBuuG1PoUxa9tBEUBKF6EKEvkZFAFIBEUqdcN3OFXix6QRCqARH6ErH88bNcN35rMVZ89IIgVA8i9CUyksl1ExDXjSAI1YcIfQkkk5rRadN1o3O4biS8UhCEKkCEvgQmQjESSSMLNp6YsehH5kTdiEUvCEI1IEJfApagAyS1TsXNj05HiSeSaa4b8dELgrD4iNCXgLUQC0bUTTCaoM7jRGsYm44SisZRCmrd8vEKgrD4iBKVwHCaRR9PaiLxJN2tvtRjwWgCr9uJUmqxpigIgpBChL4ErBh6gOmI4bZZawm9P0JQKlcKglBFiNCXQLqPPmAK/bq2GaEPSb9YQRCqCBH6Ekj30QfCpkXfVgdYrps4PrcsxAqCUB2I0JdAJou+1eehvsZluG7EohcEoYoQoS+B0ekozT43AH7Tovd6HHQ01DASiBKSNoKCIFQRIvQlMOKPsKKhFpix6L1uF+31Hob9YekXKwhCVSFCXyRaa0YCUVY0GUJvRd34PE6afR4mQ3FCsQReSZYSBKFKEKEvkqlwnGgiyYqGGgD8lkXvcdLkdTMZjJqLsWLRC4JQHZRtdiqljgB+IAHEtdbblVKtwN1AD3AEeLfWerzc96oGrIXYFY2m68by0budNHvdTIRiOB1KFmMFQaga7LLor9Zab9NabzfvfxZ4TGu9CXjMvL8ssEIrVzQaFn0gzXXT5HUTjCaYjsTFRy8IQtVQKdfNjcD3zNvfA26q0PssOFZWbOdci97jTEXiJLVUrhQEoXqwQ+g18DOl1C6l1K3m2Aqt9UkA83+nDe9TFVium86G2RZ9rctJk8+T2k4WYwVBqBbsUKPLtdYnlFKdwKNKqX2FPMk8KdwKsHbtWhumsTCMBCI4FLTXm4ux4RhetxOHQ9Hkdae2E4teEIRqoWyLXmt9wvw/BNwPXAQMKqVWApj/hzI873at9Xat9faOjo5yp7FgjASitNZ58LiMj246LQu2OU3opbuUIAjVQllCr5SqU0o1WLeBNwGvAA8Ct5ib3QI8UM77VBNToRhNXjcOswRxIqlTop5u0UvUjSAI1UK5rpsVwP1m3XUX8J9a60eUUs8DP1JKfRg4CryrzPepGqbCMepr3TgdM7XmLTeNtRibPiYIgrDYlCX0WutDwNYM46PAteW8drUSiMRprHXhTGsqYlnvDbUi9IIgVB+SGVsk/nCchloXjrRPznLdOB2KxlqXOSZRN4IgVAci9EXiD8eor3FldN0ANJnuG7HoBUGoFkToiyQQjtMwx0efvvDa7DVi6UXoBUGoFkToiyCR1ExHEzTM9dGnuWmsBVmJuhEEoVoQoS8Cq9xBLtdNo9dy3YiPXhCE6kCEvgimwjEAGmvdKKWwjHrfLNeNG4/LMetEIAiCsJiI2VkEVl2bBjOyxqkUca2pTcuCvWHrKtrM8giCIAjVgAh9EVj9Ya14eYdDQVLPsugv2dDGJRvaFmV+giAImRDXTRH4TddNvWnRu0z3jETYCIJQzYjQZ+GuXx/lpWMTs8YyuW6AWa4bQRCEakNcN1n43H0vA3Dki7+VGpsKzxZ6R8qil49REITqRSz6DGitU7cngtHUbct101Bj+Oid4roRBGEJIEKfgUg8mbr9870zpfQD4Tguh6LWbXxsDnHdCIKwBBChz0A4lkjdTvfTWwXNzLLMOM1PTyx6QRCqGRH6DASjM0JvJUmBWdCsdsYfby3GitALglDNyCpiBtKFfjIU49hYkJOTYcOir5mpOe90iutGEITqR4Q+A6E5Qv/FR/bxxL4htnQ1pCJuQCx6QRCWBiW7bpRS3Uqpx5VSe5VSryqlPm2Of0EpdVwptdv8u96+6S4MIdNH3+xzMxmK8WL/ONPRBC8fn5wl9BJeKQjCUqAchYoDf6q1fsFsEL5LKfWo+dg/aq2/XP707GPPwAQbO+sLEuVg1IiX72qsZd8pf2o8ltCz2gVaFn2NS5Y6BEGoXkpWKK31Sa31C+ZtP7AXWG3XxOxkbDrK277+S/7sx3sK2t5y3XQ11abGPGaIzSzXjUPhdTtTlr0gCEI1YospqpTqAc4HnjOHPqGU2qOUulMp1ZLlObcqpXYqpXYODw/bMY2sTIaMyJmXj08WtL21GLsyTejfet5KYLbQO5QS/7wgCFVP2UKvlKoH7gX+SGs9BdwGnAFsA04CX8n0PK317Vrr7Vrr7R0dHeVOIydRMwGqUBeL5aNf0WgI/bo2H1dubgegPj3qxqEk4kYQhKqnLKFXSrkxRP4HWuv7ALTWg1rrhNY6CXwbuKj8aZaHJdyeQoXetOjrawzr/cyuBi7d0I7H6WB1ize1ndMhFr0gCNVPyYuxykgPvQPYq7X+atr4Sq31SfPu24FXypti+ViLq4UKveW6Wd9eB8CN21bT1VTL039+Ne1pTUVE6AVBWAqUE3VzOfC7wMtKqd3m2OeBm5VS2wANHAH+oKwZ2sB0xBDuYlw3HqeDa89awRP/6yp6TMG3XDkWzV43StZhBUGockoWeq31DiCTzD1c+nQqg2XR17gKs75D0The01K3RD4TX3rneam6N4IgCNXKaZHpYzUMKdSiD0YTeAtYZJXesIIgLAVOi0yfYKS4xdhgLCG+d0EQlg2nhdBbFn2hQh+OJlKuG0EQhKXOaSH0lo8+rXFUnu3FohcEYfmw7H3013z5CQ6NTAMQSyTzbG0QjCVorF32H40gCKcJy96it0QeIJ4ozKQPi0UvCMIyYsmbre/51jNs72nhz647c95j33iib9b9eDK70CeSmsu++BhXb+lk/6CfZp8767aCIAhLiSVt0R8cDvDc4TH+9fGD8x67Y8dhvvTI/llj8eSM6yYaT/K6Lz3Owy8bSbyHRwIMTkX44fPHANgzUFgBNEEQhGpnSVv0VungjoYa7tk1wN3PH+XHH72M7zx9iL/5773zto8nNGPTUW6+/Vk+cuV6jo4F+dgPXqDO45yXGFXrXtLnQEEQhBRKFxqKUkG2b9+ud+7cWdJz33v7Mzx7aCx1f3Wzl+MTobLn1N3q5enPXFP26wiCIFQKpdQurfX2fNstebP1ucNjs+5bIn/zRWtn1ZPvafMV9HoX9bRy5aZ2/v1DF9s3SUEQhEVkSbtuAO77w8t4+zd+BcCGjjpWNtWyorGW91+8lr9861l84/GDfOKajdyx4zD/8NMZn/0nr9nIho46tqxo5P3feZbxYIwN7XX86KOXLtauCIIgVIQl77oB+NaTB7nsjHbOXdOUdZvpSJzP3/8yR0amedM5XXz86o2pxwkiA6YAAAXgSURBVLTWfO2xPm7YupINHfUlz0MQBGEhKdR1syyEXhAE4XTktPHRC4IgCLkRoRcEQVjmVEzolVJvVkrtV0r1KaU+W6n3EQRBEHJTEaFXSjmBfwXeApyN0V7w7Eq8lyAIgpCbSln0FwF9WutDWuso8EPgxgq9lyAIgpCDSgn9auBY2v0Bc0wQBEFYYCol9Jk6Zs+K41RK3aqU2qmU2jk8PFyhaQiCIAiVEvoBoDvt/hrgRPoGWuvbtdbbtdbbOzo6KjQNQRAEoSIJU0opF3AAuBY4DjwPvE9r/WqW7YeB/hLfrh0YKfG5S5nTcb9Px32G03O/T8d9huL3e53WOq+lXJFaN1rruFLqE8BPASdwZzaRN7cv2aRXSu0sJDNsuXE67vfpuM9weu736bjPULn9rlhRM631w8DDlXp9QRAEoTAkM1YQBGGZsxyE/vbFnsAicTru9+m4z3B67vfpuM9Qof2uiuqVgiAIQuVYDha9IAiCkIMlLfTLqXCaUqpbKfW4UmqvUupVpdSnzfFWpdSjSqle83+LOa6UUl8z932PUuqCtNe6xdy+Vyl1y2LtU6EopZxKqReVUg+Z99crpZ4z53+3UspjjteY9/vMx3vSXuNz5vh+pdR1i7MnhaOUalZK3aOU2mce80uX+7FWSv2x+d1+RSl1l1Kqdjkea6XUnUqpIaXUK2ljth1bpdSFSqmXzed8TSmVKUF1NlrrJfmHEbZ5ENgAeICXgLMXe15l7M9K4ALzdgNGHsLZwJeAz5rjnwX+3rx9PfA/GFnIlwDPmeOtwCHzf4t5u2Wx9y/Pvv8J8J/AQ+b9HwHvNW9/E/hD8/bHgG+at98L3G3ePts8/jXAevN74Vzs/cqzz98DPmLe9gDNy/lYY5RAOQx4047xB5fjsQZeB1wAvJI2ZtuxBX4NXGo+53+At+Sd02J/KGV8mJcCP027/zngc4s9Lxv37wHgjcB+YKU5thLYb97+FnBz2vb7zcdvBr6VNj5ru2r7w8iafgy4BnjI/PKOAK65xxkjL+NS87bL3E7NPfbp21XjH9Boip6aM75sjzUz9a9azWP3EHDdcj3WQM8cobfl2JqP7Usbn7Vdtr+l7LpZtoXTzMvU84HngBVa65MA5v9Oc7Ns+7/UPpd/Aj4DJM37bcCE1jpu3k+ff2rfzMcnze2X2j5vAIaBfzNdVt9RStWxjI+11vo48GXgKHAS49jtYvkfawu7ju1q8/bc8ZwsZaHPWzhtKaKUqgfuBf5Iaz2Va9MMYzrHeNWhlHorMKS13pU+nGFTneexJbPPJi6MS/vbtNbnA9MYl/PZWPL7bfqkb8Rwt6wC6jD6VcxluR3rfBS7nyXt/1IW+ryF05YaSik3hsj/QGt9nzk8qJRaaT6+Ehgyx7Pt/1L6XC4H3qaUOoLRs+AaDAu/WRn1kmD2/FP7Zj7eBIyxtPYZjPkOaK2fM+/fgyH8y/lYvwE4rLUe1lrHgPuAy1j+x9rCrmM7YN6eO56TpSz0zwObzFV7D8aCzYOLPKeSMVfO7wD2aq2/mvbQg4C14n4Lhu/eGv+AuWp/CTBpXhL+FHiTUqrFtKLeZI5VHVrrz2mt12itezCO3y+01u8HHgfeaW42d5+tz+Kd5vbaHH+vGamxHtiEsWBVlWitTwHHlFJbzKFrgd+wjI81hsvmEqWUz/yuW/u8rI91GrYcW/Mxv1LqEvNz/EDaa2VnsRctylzwuB4jOuUg8BeLPZ8y9+UKjEuwPcBu8+96DL/kY0Cv+b/V3F5htGs8CLwMbE97rQ8Bfebf7y32vhW4/1cxE3WzAePH2wf8GKgxx2vN+33m4xvSnv8X5mexnwKiEBb7D9gG7DSP908wIiuW9bEG/hrYB7wC/DtG5MyyO9bAXRjrEDEMC/zDdh5bYLv5GR4Evs6cRf1Mf5IZKwiCsMxZyq4bQRAEoQBE6AVBEJY5IvSCIAjLHBF6QRCEZY4IvSAIwjJHhF4QBGGZI0IvCIKwzBGhFwRBWOb8/2HpvPmEKIL/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(H.history['nb_steps'],H.history['episode_reward'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (spinningup)",
   "language": "python",
   "name": "spinningup"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
